import requests
from lxml import etree
from secondUrl import secondUrl


url_context = [
    'http://www.biqu5200.net//xuanhuanxiaoshuo/','http://www.biqu5200.net//xiuzhenxiaoshuo/',
    'http://www.biqu5200.net//dushixiaoshuo/','http://www.biqu5200.net//chuanyuexiaoshuo/','http://www.biqu5200.net//wangyouxiaoshuo/',
    'http://www.biqu5200.net//kehuanxiaoshuo/','http://www.biqu5200.net//jushixiaoshuo/','http://www.biqu5200.net//yanqingxiaoshuo/'
]
url = 'http://www.biqu5200.net/'
bookUrl = [];

headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
}
bookContent =  []
for item in url_context:
    response = requests.get(url=item,headers=headers).text
    html = etree.HTML(response)
    for index in range(1,30):
        u1 = secondUrl(index)
        parseHtml = html.xpath(u1)[0]
        ToUrl =url + parseHtml
        print(parseHtml)
        bookContent.append(ToUrl)
        request = requests.get(url=ToUrl,headers=headers).text
        resp = etree.HTML(request)
        # img = resp.xpath('//*[@id="fmimg"]/img/@src')[0]
        # title = resp.xpath('//*[@id="info"]/h1/text()')[0]
        # author = resp.xpath('//*[@id="info"]/p[1]/text()')[0]
        # info = resp.xpath('//*[@id="intro"]/p/text()')[0]
        # print(author)
        # print(title)
        # print(ToUrl)
        # print(info)
        # zj_context = resp.xpath('//*[@id="list"]/dl/dd/a/text()')[0]
        # zj_href = resp.xpath('//*[@id="list"]/dl//dd/a/@href')[0]
        # print(zj_context)
        # print(zj_href)

print(bookContent)
